{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# A MLP Classifier for Fashion-mnist\n",
    "\n",
    "You need to implement several parts of mlp.\n",
    "\n",
    "- layers\n",
    "- activations\n",
    "- forward propagation\n",
    "- backward propagation\n",
    "- optimization(e.g. SGD)\n",
    "\n",
    "You should get an accuracy of more than **88.5** for development dataset and **88.0** for test dataset with proper strategy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "class relu:\n",
    "    def __init__(self):\n",
    "        pass\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        '''\n",
    "        forward propagation through ReLU\n",
    "        \n",
    "        Prameters\n",
    "        --------\n",
    "        inputs: numpy.ndarray\n",
    "            inputs for forward propagation\n",
    "        \n",
    "        Return\n",
    "        ------\n",
    "        outputs: numpy.ndarray\n",
    "            outputs through ReLU\n",
    "        '''\n",
    "        # Your code here! You need to implement forward propagation through ReLU.\n",
    "        \n",
    "    def backward(self, inputs, outputs, fgrad):\n",
    "        '''\n",
    "        backward propagation through ReLU\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        inputs: numpy.ndarray\n",
    "            input for ReLU function\n",
    "        outputs: numpy.ndarray\n",
    "            output of ReLU function\n",
    "        fgrag: numpy.ndarray\n",
    "            d(Loss)/d(outputs)\n",
    "            \n",
    "        Return\n",
    "        ------\n",
    "        grad: numpy.ndarray\n",
    "            d(Loss)/d(inputs)\n",
    "        '''\n",
    "        # Your code here! You need to implement backward algorithm through ReLU.\n",
    "        \n",
    "\n",
    "class softmax:\n",
    "    def __init__(self):\n",
    "        pass\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        '''\n",
    "        forward propagation through softmax\n",
    "        \n",
    "        Prameters\n",
    "        --------\n",
    "        inputs: numpy.ndarray\n",
    "            inputs for forward propagation\n",
    "        \n",
    "        Return\n",
    "        ------\n",
    "        outputs: numpy.ndarray\n",
    "            outputs through softmax\n",
    "        '''\n",
    "        # Your code here! You need to implement forward algorithm through softmax.\n",
    "        \n",
    "    def backward(self, inputs, outputs, fgrad):\n",
    "        '''\n",
    "        forward propagation through softmax\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        inputs: numpy.ndarray\n",
    "            input for softmax function\n",
    "        outputs: numpy.ndarray\n",
    "            output of softmax function\n",
    "        fgrag: numpy.ndarray\n",
    "            d(Loss)/d(outputs)\n",
    "        \n",
    "        Return\n",
    "        ------\n",
    "        grad: numpy.ndarray\n",
    "            d(Loss)/d(inputs)\n",
    "        '''\n",
    "        # Your code here! You need to implement backward algorithm through softmax.\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class FullyConnected:\n",
    "    def __init__(self, dim_in, dim_out, batch_size, activation):\n",
    "        # initialization according to He et al.(2015)\n",
    "        self.W = np.random.randn(dim_in, dim_out).astype(np.float32) \\\n",
    "                 * np.sqrt(2.0/(dim_in))\n",
    "        self.b = np.zeros([dim_out]).astype(np.float32)\n",
    "        self.batch_size = batch_size\n",
    "        self.activation = activation\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        '''\n",
    "        Forward propagation through a fully connected layer\n",
    "        \n",
    "        Prameters\n",
    "        --------\n",
    "        inputs: numpy.ndarray\n",
    "            inputs for forward propagation\n",
    "        \n",
    "        Return\n",
    "        ------\n",
    "        outputs: numpy.ndarray\n",
    "            outputs through a fully connected layer\n",
    "        '''\n",
    "        # Your code here! You need to implement backward algorithm through fully connected layers.\n",
    "\n",
    "\n",
    "    def backward(self, grad):\n",
    "        '''\n",
    "        Backward propagation through a fully connected layer\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        grad: numpy.ndarray\n",
    "            d(Loss)/d(self.outputs)\n",
    "        \n",
    "        Return\n",
    "        ------\n",
    "        grad: numpy.ndarray\n",
    "            d(Loss)/d(self.inputs)\n",
    "        '''\n",
    "        activ_grad = self.activation.backward(self.outputs,\n",
    "                                              self.outputs_act, grad)\n",
    "        # Your code below! You need to implement backward algorithm through fully connected layers.        \n",
    "        self.grad_b = # Your code here! You need to calculate d(Loss)/d(self.b)\n",
    "        self.grad_W = # Your code here! You need to calculate d(Loss)/d(self.W)\n",
    "        grad_inputs = # Your code here! You need to calculate d(Loss)/d(self.inputs)\n",
    "        return grad_inputs\n",
    "\n",
    "    def update(self, lr):\n",
    "        '''\n",
    "        update W and B according to learning rate and gradient\n",
    "        Parameters\n",
    "        ----------\n",
    "        lr: integer\n",
    "            learning rate.\n",
    "        '''\n",
    "        # Your code here! You need to update W and B according to learning rate and gradient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "\n",
    "logging.basicConfig(level=logging.INFO,\n",
    "                    format='%(asctime)-15s %(levelname)s: %(message)s')\n",
    "LOG = logging.getLogger('mlp-fashion-mnist')\n",
    "# np.random.seed(233)\n",
    "\n",
    "\n",
    "def evaluate(inputs, y):\n",
    "    for layer in FCs:\n",
    "        outputs = layer.forward(inputs)\n",
    "        inputs = outputs\n",
    "    outputs = np.argmax(outputs, axis=1)\n",
    "    precision = float(np.mean(outputs == y))\n",
    "    return precision\n",
    "\n",
    "\n",
    "def load_mnist(path, kind='train'):\n",
    "    import os\n",
    "    import gzip\n",
    "    import numpy as np\n",
    "\n",
    "    \"\"\"Load MNIST data from 'path'\"\"\"\n",
    "    labels_path = os.path.join(path,\n",
    "                               '%s-labels-idx1-ubyte.gz'\n",
    "                               % kind)\n",
    "    images_path = os.path.join(path,\n",
    "                               '%s-images-idx3-ubyte.gz'\n",
    "                               % kind)\n",
    "\n",
    "    with gzip.open(labels_path, 'rb') as lbpath:\n",
    "        labels = np.frombuffer(lbpath.read(), dtype=np.uint8,\n",
    "                               offset=8)\n",
    "\n",
    "    with gzip.open(images_path, 'rb') as imgpath:\n",
    "        images = np.frombuffer(imgpath.read(), dtype=np.uint8,\n",
    "                               offset=16).reshape(len(labels), 784)\n",
    "\n",
    "    return images, labels\n",
    "\n",
    "\n",
    "class params:\n",
    "    def __init__(self):\n",
    "        self.batch_size = 200\n",
    "        self.epochs = 25\n",
    "        self.lr = 0.1\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    opts = params()\n",
    "\n",
    "    X_train, y_train = load_mnist('../fashion-mnist/data/fashion',\n",
    "                                  kind='train')\n",
    "    X_test, y_test = load_mnist('../fashion-mnist/data/fashion',\n",
    "                                kind='t10k')\n",
    "    X_train = X_train.astype(np.float32) / 255\n",
    "    X_test = X_test.astype(np.float32) / 255\n",
    "    r = np.random.permutation(len(y_train))\n",
    "    X_train = X_train[r]\n",
    "    y_train = y_train[r]\n",
    "    X_dev = X_train[:12000]\n",
    "    y_dev = y_train[:12000]\n",
    "    X_train = X_train[10000:]\n",
    "    y_train = y_train[10000:]\n",
    "\n",
    "    LOG.info(\"finish data preprocessing.\")\n",
    "\n",
    "    FCs = [FullyConnected(784, 256, opts.batch_size, relu()),\n",
    "           FullyConnected(256, 128, opts.batch_size, relu()),\n",
    "           FullyConnected(128, 64, opts.batch_size, relu()),\n",
    "           FullyConnected(64, 10, opts.batch_size, softmax())]\n",
    "\n",
    "    LOG.info(\"finish initialization.\")\n",
    "\n",
    "    n_samples = len(y_train)\n",
    "    order = np.arange(n_samples)\n",
    "    best_accuracy, test_accuracy = 0, 0\n",
    "    for epochs in range(0, opts.epochs):\n",
    "        np.random.shuffle(order)\n",
    "        cost = 0.\n",
    "        for batch_start in range(0, n_samples, opts.batch_size):\n",
    "            batch_end = batch_start + opts.batch_size if batch_start \\\n",
    "                        + opts.batch_size < n_samples else n_samples\n",
    "            batch_id = order[batch_start: batch_end]\n",
    "            xs, ys = X_train[batch_id], y_train[batch_id]\n",
    "            inputs = xs\n",
    "            for layer in FCs:\n",
    "                outputs = layer.forward(inputs)\n",
    "                inputs = outputs\n",
    "            cost += -np.mean(np.log(outputs) * np.eye(10)[ys])\n",
    "            grad = # Your code here! You need to calculate d(Loss)/d(outputs)\n",
    "            for layer in range(len(FCs)-1, -1, -1):\n",
    "                grad = FCs[layer].backward(grad)\n",
    "            for layer in FCs:\n",
    "                layer.update(opts.lr)\n",
    "        accuracy = evaluate(X_dev, y_dev)\n",
    "        LOG.info(\"iteration {0}, cost = {1}, dev_accuracy = {2}\".format(\n",
    "            epochs, cost, precision))\n",
    "        if accuracy > best_accuracy:\n",
    "            best_accuracy = accuracy\n",
    "            test_accuracy = evaluate(X_test, y_test)\n",
    "            LOG.info(\"New best achived. test_accuracy = {0}\"\n",
    "                     .format(test_accuracy))\n",
    "    LOG.info(\"Training finished. dev_accuracy = {0}, test_accuracy = {1}\"\n",
    "             .format(best_accuracy, test_accuracy))\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
